package com.andnnl.file;

import cn.hutool.core.collection.IterUtil;
import cn.hutool.core.util.NumberUtil;
import cn.hutool.core.util.ReUtil;
import com.andnnl.fileanalysis.AnalysisListener;
import com.andnnl.fileanalysis.AnalysisParam;
import com.andnnl.fileanalysis.CountListener;
import com.andnnl.fileanalysis.FileAnalysisManage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import java.util.stream.Collectors;


/**
 * 域名和IP提取
 * Created by chenss on 2024/1/10.
 */
public class HttpWhiteDomainIPFromContentTest {
    private static final Logger logger = LoggerFactory.getLogger(HttpWhiteDomainIPFromContentTest.class);

    public static void main(String[] args) throws IOException {
//        test1();
        test2();
        return;
    }

    public static void test2() {
        Map<String, Boolean> map = new HashMap<>();
        File file = new File("C:\\Users\\Administrator\\Desktop\\contetxt_202312.csv");
        Pattern regexIp = Pattern.compile("((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)", Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE | Pattern.MULTILINE);
        Pattern regexDomain = Pattern.compile("((http://|https://){0,1}([\\w\\-~]+\\.)+([a-z]){2,6})(/.*)?", Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE | Pattern.MULTILINE);
        AnalysisListener listener = rowList -> {
//            logger.info("batch size:" + rowList.size());
            for (List<String> strings : rowList) {
                if (strings.size() < 3) continue;
//                logger.info(strings.toString());
                final String eprId = strings.get(0);
                final String uid = strings.get(1).trim();
                final String domain = strings.get(2).replaceAll("\n", "");
                if(!NumberUtil.isInteger(eprId))continue;
                if(uid.length() > 10)continue;
                if (uid.indexOf(",") != -1) continue;
                if (domain.toLowerCase().indexOf("apple") != -1) continue;//忽略苹果
                if (domain.toLowerCase().indexOf("www.ruijie.com.cn") != -1) continue;//忽略www.ruijie.com.cn
                List<String> domainList = ReUtil.findAll(regexDomain, domain, 1);//正则获取域名列表
                List<String> ipList = ReUtil.findAll(regexIp, domain, 0);//正则获取域名列表
                domainList = domainList.stream()
                        .filter(d -> d.indexOf(".htm") == -1)
                        .filter(d -> d.indexOf(".pdf") == -1)
                        .filter(d -> d.indexOf(".xml") == -1)
//                        .filter(d -> d.indexOf("oracle.jdbc") == -1)
                        .distinct().collect(Collectors.toList());
//                logger.info("domainList:" + domainList);
//                System.out.println();
                //白名单类型(0:域名,1:IP)
                if (IterUtil.isNotEmpty(domainList)) {
                    for (String http : domainList) {
                        final String x = eprId + "," + uid + "," + 0 + "," + http;
                        if (!map.containsKey(x)) {
                            map.put(x, Boolean.TRUE);
//                            System.out.println(x);
                        }
                    }
                }
                if (IterUtil.isNotEmpty(ipList)) {
                    for (String ip : ipList) {
                        final String x = eprId + "," + uid + "," + 1 + "," + ip;
                        if (!map.containsKey(x)) {
                            map.put(x, Boolean.TRUE);
//                            System.out.println(x);
                        }

                    }
                }
            }
        };
        AnalysisParam param = AnalysisParam.build()
                .setBatchSize(1000)//读到一批后调用一次invokeBatch
                .setDelHeadRows(1)//删除头部行数，空行不算
//                .setExt("txt")//指定解析的文件格式
//                .setCharset("UTF-8")//指定解析的文件编码
//                .setCharset("GBK")//指定解析的文件编码
//                .setSplitLimit(3)//txt分割2列
                .setMaxRow(0);//限制读取行数,0:不限制
//        CountListener countlistener = new CountListener();
//        FileAnalysisManage.analysis(file, countlistener, param);
//        logger.info("listener.count:" + countlistener.getCount());
        FileAnalysisManage.analysis(file, listener, param);
        System.out.println("******************************************************");
        System.out.println(map.size());
        map.keySet().forEach(k->{
            System.out.println(""+k);
        });
        System.out.println("546,szzgyzcx,0,http://www.ruijie.com.cn");
    }

}
